import requests
from lxml import etree
import re

total = 0


def get_ip():
    # 快代理后台生成的api链接   5s访问一次
    api_url = "https://dps.kdlapi.com/api/getdps/?secret_id=olrejp12b94qknshw4be&signature=m1y23gc6dg9ivkljppeovaaprqkqdgdh&num=1&pt=1&sep=1"
    # 获取API接口返回的代理IP
    proxy_ip = requests.get(api_url).text
    print("获取新IP地址：", proxy_ip)
    # 订单生成的账户与密码
    username = "d2356162166"
    password = "l7zee1jn"
    # 代理
    proxies = {
        "http": f"http://{username}:{password}@{proxy_ip}",
        "https": f"http://{username}:{password}@{proxy_ip}",
    }
    return proxies


def requests_with_proxy(url, headers=None):
    global proxies
    for i in range(100):
        try:
            response = requests.get(url, headers=headers, proxies=proxies, allow_redirects=False)
        except requests.exceptions.ProxyError as e:
            print("ProxyError:", e)
            proxies = get_ip()
            continue
        except requests.exceptions.ReadTimeout as e:
            print("ReadTimeout:", e)
            proxies = get_ip()
            continue
        if response.status_code == 302:
            proxies = get_ip()
        elif response.status_code == 200:
            return response
    return None


proxies = get_ip()
url = f"https://zz.lianjia.com/zufang/pg1/#contentList"
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
}
response = requests_with_proxy(url, headers=headers)
result = response.content.decode()
root = etree.HTML(result)
# 获取城市的所有区
area_list = root.xpath("//div[@id='filter']/ul[2]/li[@class='filter__item--level2  ']/a")
for area in area_list:
    area_name = area.xpath("./text()")[0]
    area_url = "https://zz.lianjia.com" + area.xpath("./@href")[0]
    # 计算不同区的租房数量
    response = requests_with_proxy(area_url, headers=headers)
    html_str = response.content.decode()
    area_total = re.search('<span class="content__title--hl">(\d+)</span>', html_str).group(1)
    area_total = int(area_total)
    if area_total > 3000:
        # 计算区域下一级商圈的租房数量
        root = etree.HTML(html_str)
        shopping_list = root.xpath("//div[@id='filter']/ul[4]/li[@class='filter__item--level3  ']/a")
        for shopping in shopping_list:
            shopping_name = shopping.xpath("./text()")[0]
            shopping_url = "https://zz.lianjia.com" + shopping.xpath("./@href")[0]
            # 计算商圈下的租房数量
            response = requests_with_proxy(shopping_url, headers=headers)
            html_str = response.content.decode()
            shopping_total = re.search('<span class="content__title--hl">(\d+)</span>', html_str).group(1)
            shopping_total = int(shopping_total)
            total += shopping_total
            print(area_name, shopping_name, shopping_total, total)
    else:
        total += area_total
        print(area_name, area_total, total)
